import random
import requests
from util.ip import get_proxies, get_proxies_pull

proxies_pull = get_proxies_pull()

urls = [f"https://httpbin.org/get?age={i}" for i in range(100)]

while len(urls):
    try:
        url = urls.pop()
        proxy = random.choice(proxies_pull)
        try:
            response = requests.get(url, proxies=proxy)
            print(response.json())
        except requests.exceptions.ProxyError as e:
            print("IP代理出错, 重新获取新的IP代理")
            proxies_pull.remove(proxy)
            proxies_pull.append(get_proxies())
            urls.append(url)
        except Exception as e:
            print("未知错误:", e)
            urls.append(url)
    except StopIteration as e:
        print("爬取结束....")






